import requests
from requests_html import HTMLSession
import time
import os
session = HTMLSession()

allImgList=[]

def get_IndexPage(url):
    response = session.get(url)
    pagediv = response.html.find('div.NewPages', first=True)
    pageli = pagediv.find('li')

    for li in pageli:
        if(not li.find('a',first=True) is None):
            if(li.find('a',first=True).text == '下一页'):
                newUrl = mainUrl+li.find('a',first=True).attrs['href']
                returnList= get_img_list(newUrl)
                allImgList.append(returnList)
                get_IndexPage(newUrl)



def get_img_list(url):
    response = session.get(url)
    content = response.html.find('div.TypeList', first=True)
    li_list = content.find('li')
    imgList = []
    for li in li_list:
        url = li.find('a.TypeBigPics',first=True).attrs['href']
        imgList.append(url)
    return imgList


def get_SecPage(url):
    response = session.get(url)
    pagediv = response.html.find('div.NewPages', first=True)
    pageli = pagediv.find('li')
    imgUrl=os.path.split(url)[0]
    for li in pageli:
        if(not li.find('a',first=True) is None):
            if(li.find('a',first=True).text == '下一页'):
                if(not li.find('a',first=True).attrs['href']=="#"):
                    newUrl = imgUrl+'/'+li.find('a',first=True).attrs['href']
                    get_img_detail(newUrl)
                    get_SecPage(newUrl)
                
    # time.sleep(2)


def get_img_detail(url):
    response = session.get(url)
    content = response.html.find('div.ImageBody', first=True)
    imgList = content.find('img')
    for img in imgList:
        imgurl = img.attrs['src']
        filename = img.attrs['alt']
        save_img(imgurl, filename)




def save_img(imgurl,filename):
    header = {"user-agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.26 Safari/537.36 Core/1.63.5702.400 QQBrowser/10.2.1893.400"}
    img_respose = requests.get(imgurl,header)
    t = int(round(time.time()*1000))
    savePath = ('F:/Desktop/Work/05Img-Spider/www.umei.cc-xiezhen/%s/'%filename)
    havedir= os.path.exists(savePath)
    if not havedir:
        os.makedirs(savePath)
    f = open(savePath+'%d.jpg' %t, 'ab')
    f.write(img_respose.content)
    f.close()
    time.sleep(0.1)


if __name__ == '__main__':
    mainUrl = 'http://www.umei.cc'
    url = 'http://www.umei.cc/tags/xiezhen.htm'
    #typeclass = 'TypeList'
    #savePath = ''

    get_IndexPage(url)
    for urlList in  allImgList:
        for url in urlList:
            get_SecPage(url)
    # for url in imgList:
    #     get_img_detail(url)
    #     time.sleep(2)


